import io
import sys
import pymysql
import requests
from lxml import etree

# 定义网址
start_url = 'https://top.baidu.com/board?tab=realtime'
# 定义浏览器标识和相应cookie，F12+network获取
# 添加请求头

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36 Edg/96.0.1054.29',
    'cookie': "BIDUPSID=4BD5E9A9C6587825F52D984062B0D4D1; PSTM=1634540024; BAIDUID=A3A87D14FFBE905FFD355F20D2E5CDB2:FG=1; MCITY=-300%3A; BAIDUID_BFESS=A3A87D14FFBE905FFD355F20D2E5CDB2:FG=1; BDORZ=FAE1F8CFA4E8841CC28A015FEAEE495D"
}  #字典变量

sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8')  # 改变标准输出的默认编码
# 保存到数据库
def saveDAO(db,number,topic,hot,href):
    cursor = db.cursor()
    hotsql = """insert into baidutop(ranking,topic,hot,url,daydate,mindate) values (%s,%s,%s,%s,curdate(),curtime())"""
    cursor.execute(hotsql,(number,topic,hot,href))
    cursor.close()

# 进行requests请求
def get(url):
    response = requests.get(url, headers=headers)
    if response. status_code == 200:
        print("网页请求成功")
        parse(url)
    else:
        print("网页请求失败了")

# 将百度热搜的数据给爬下来
def parse(url):
    response = requests.get(url, headers=headers)
    selector = etree.HTML(response.text)
    # 获取百度热搜信息
    number = [each.text for each in selector.cssselect('#sanRoot > main > div.container.right-container_2EFJr > div > div:nth-child(2) > div > a > div.index_1Ew5p')]
    topic = [each.text for each in selector.cssselect('#sanRoot > main > div.container.right-container_2EFJr > div > div:nth-child(2) > div > div.content_1YWBm > a > div.c-single-text-ellipsis')]
    hot = [each.text for each in selector.cssselect('#sanRoot > main > div.container.right-container_2EFJr > div > div:nth-child(2) > div > div.trend_2RttY.hide-icon > div.hot-index_1Bl1a')]
    href = [each.get("href") for each in selector.cssselect('#sanRoot > main > div.container.right-container_2EFJr > div > div:nth-child(2) > div > div.content_1YWBm > a')]
    # 连接数据库
    db = pymysql.connect(host="localhost", user="root", password="root", database="hottop")
    # 因为id自动递增，让自动增长从1开始，已经有了就跳过，让id连续
    cursor = db.cursor()
    str = 'ALTER TABLE baidutop AUTO_INCREMENT = 1'
    cursor.execute(str)
    cursor.close()
    db.commit()
    # 保存并提交到数据库
    try:
        for i in range(len(number)):
#             print(number[i], topic[i], hot[i], href[i])
            saveDAO(db, number[i], topic[i], hot[i], href[i])
    except BaseException as ex:
        print(ex)
    else:
        db.commit()
        db.close()
    response.close()



#程序从此处开始运行
if __name__ == "__main__":
    get(start_url)
